# -*- coding: utf-8 -*-
# 爬取研究生院
# http://pg.njupt.edu.cn/s/26/t/775/p/1/c/8382/list.htm
from bs4 import BeautifulSoup
import requests

import pymongo
import config

mongo_connection = pymongo.Connection(config.MongoHOST, config.MongoPORT)
db=mongo_connection.example
jwc_test=db.secondtest

__author__ = 'insect'

PgBaseUrl = "http://pg.njupt.edu.cn"

class DownloadPgForm():
    def __init__(self):
        self.base_url = "http://pg.njupt.edu.cn/s/26/t/775/p/1/c/8382/d/"
        self.num_list = ['8446/list.htm', '8383/list.htm', '8383/i/2/list.htm',
                         '8447/list.htm', '8384/list.htm', '8384/i/2/list.htm',
                         '8385/list.htm', '8450/list.htm']
        self.result_list = []

    def get_page(self, url):
        response = requests.get(url)
        response.encoding = "utf-8"
        html = BeautifulSoup(response.text)
        return html

    def make_url(self, number):
        return self.base_url + number

    def parse_form_list(self, html):
        soup = html
        the_list = soup.find_all("table", class_="columnStyle")
        if len(the_list) == 0:
            pass
        for item in the_list:
            if str(item.a['href'])[-4:] == '.doc':
                the_dic = {
                    "title": item.font.string,
                    "url": (PgBaseUrl + item.a['href']),
                    "identify": "graduate student form"
                }
                self.result_list.append(the_dic)
                # print item.font.string
                # print item.a['href']
                continue
            else:
                form_url = PgBaseUrl + str(item.a['href'])
                self.parse_real_html(self.get_page(form_url), form_url)

    def parse_real_html(self, html, form_url):
        soup = html
        title = soup.title.string
        if title[-1:] == '\n':
            title = title[:-1]
        if title[-2:] == '\r\n':
            title = title[:-2]
        # print title
        try:
            table_list = soup.find('td', class_="content")
            if table_list is None:
                the_dic = {
                    "title": title,
                    "url": form_url,
                    "identify": "pg form"
                }
                self.result_list.append(the_dic)
            else:
                a_list = table_list.find_all("a")
                for item in a_list:
                    the_dic = {
                        "title": item.string,
                        "url": (PgBaseUrl + item['href']),
                        "identify": "pg form",
                    }
                    self.result_list.append(the_dic)
                # the_dic = {"title": soup.title.string,
                #            "url": the_url_list,
                #            "identify": "graduate student form"}
                # self.result_list.append(the_dic)
                    # print item.string
                    # print item['href']
        except Exception:
            pass

    def get(self):
        for key in self.num_list:
            self.parse_form_list(self.get_page(self.make_url(key)))

    def get_result(self):
        self.get()
        return self.result_list

if __name__ == '__main__':
    test = DownloadPgForm()
    result = test.get_result()
    for item in result:
        jwc_test.insert(item)
    # print count
    # search_result = jwc_test.find({"title": {'$regex': '保格生'}})
    # print search_result
    # new_search_result = []
    # repetition_list = []
    # for item in search_result:
    #     if item['url'] not in repetition_list:
    #         new_search_result.append(item)
    #         repetition_list.append(item['url'])
    #     else:
    #         continue
    # print new_search_result
    # for item in new_search_result:
    #     print item['title']
    #     print item['url']
    # test.get()